{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "12f4dced",
   "metadata": {},
   "source": [
    "双模型使用两个不同的模型计算value和target,缓解了自举造成的过高估计."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "91162df8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAATcklEQVR4nO3df2xT5b8H8He7rWVsO60brnXZekeiV1z4oQ7Yjv6hgcrEhYguuWqITkIwYscVR4guUfydEbyJP3H8o8A/iJnJ9Lrgj9nJiKEwqS4Zm+zq94vZLq6tsO9Ot8G6H33uH2bnWhm4jq5PS9+v5CSe8zw9/ZxH+u45z2lXgxBCgIhIAqPsAogodTGAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGmkBtGfPHhQXF2PevHkoKytDe3u7rFKISBIpAfTxxx+jtrYWL774In744QcsW7YMFRUVCAQCMsohIkkMMr6MWlZWhhUrVuC9994DAITDYRQVFWHr1q147rnn4l0OEUmSHu8nHBsbg9frRV1dnb7NaDTC6XTC4/FM+5hQKIRQKKSvh8NhDAwMIC8vDwaDYc5rJqLoCCEwNDSEgoICGI2Xv9CKewCdO3cOk5OTsNlsEdttNhtOnz497WPq6+vx8ssvx6M8Ioqhvr4+FBYWXrY97gE0G3V1daitrdXXNU2Dw+FAX18fFEWRWBkRTScYDKKoqAg5OTlX7Bf3AFqwYAHS0tLg9/sjtvv9ftjt9mkfYzabYTabL9muKAoDiCiB/d0USdzvgplMJpSWlsLtduvbwuEw3G43VFWNdzlEJJGUS7Da2lpUV1dj+fLlWLlyJd566y2MjIxg48aNMsohIkmkBNBDDz2E33//HTt37oTP58Ott96KL7/88pKJaSK6tkn5HNDVCgaDsFgs0DSNc0BECWimr1F+F4yIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSRB1AR48exbp161BQUACDwYBPP/00ol0IgZ07d+KGG25AZmYmnE4nfv7554g+AwMD2LBhAxRFgdVqxaZNmzA8PHxVB0JEySfqABoZGcGyZcuwZ8+eadt3796Nd955B3v37sWJEyeQlZWFiooKjI6O6n02bNiArq4utLS0oLm5GUePHsUTTzwx+6MgouQkrgIA0dTUpK+Hw2Fht9vFG2+8oW8bHBwUZrNZfPTRR0IIIbq7uwUA8f333+t9vvjiC2EwGMTZs2dn9LyapgkAQtO0qymfiObITF+jMZ0DOnPmDHw+H5xOp77NYrGgrKwMHo8HAODxeGC1WrF8+XK9j9PphNFoxIkTJ6bdbygUQjAYjFiIKPnFNIB8Ph8AwGazRWy32Wx6m8/nQ35+fkR7eno6cnNz9T5/VV9fD4vFoi9FRUWxLJuIJEmKu2B1dXXQNE1f+vr6ZJdERDEQ0wCy2+0AAL/fH7Hd7/frbXa7HYFAIKJ9YmICAwMDep+/MpvNUBQlYiGi5BfTAFq4cCHsdjvcbre+LRgM4sSJE1BVFQCgqioGBwfh9Xr1Pq2trQiHwygrK4tlOUSU4NKjfcDw8DB++eUXff3MmTPo6OhAbm4uHA4Htm3bhtdeew033XQTFi5ciBdeeAEFBQVYv349AOCWW27Bvffei82bN2Pv3r0YHx9HTU0NHn74YRQUFMTswIgoCUR7e+3bb78VAC5ZqqurhRB/3Ip/4YUXhM1mE2azWaxevVr09PRE7OP8+fPikUceEdnZ2UJRFLFx40YxNDQU81t8RCTHTF+jBiGEkJh/sxIMBmGxWKBpGueDiBLQTF+jSXEXjIiuTQwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISJqof5aHKFbGLw4h+L/d+rrBmAbrvy2FMd0ksSqKJwYQSTM66MM/Wz/Q19NMmVj8Hy/BxABKGbwEI2mECMsugSRjAJE8yfeTdBRjDCCSJgl/E5NijAFE8jCAUh4DiCRiAKU6BhBJw0loiiqA6uvrsWLFCuTk5CA/Px/r169HT09PRJ/R0VG4XC7k5eUhOzsbVVVV8Pv9EX16e3tRWVmJ+fPnIz8/Hzt27MDExMTVHw0lF16CpbyoAqitrQ0ulwvHjx9HS0sLxsfHsWbNGoyMjOh9nnnmGXz++edobGxEW1sbfvvtNzz44IN6++TkJCorKzE2NoZjx47hwIED2L9/P3bu3Bm7o6KkwElogrgKgUBAABBtbW1CCCEGBwdFRkaGaGxs1Pv89NNPAoDweDxCCCEOHz4sjEaj8Pl8ep+GhgahKIoIhUIzel5N0wQAoWna1ZRPkg3880fRvnezvng//E8RGh6QXRbFwExfo1c1B6RpGgAgNzcXAOD1ejE+Pg6n06n3WbRoERwOBzweDwDA4/FgyZIlsNlsep+KigoEg0F0dXVN+zyhUAjBYDBioeQnOAmd8mYdQOFwGNu2bcOdd96JxYsXAwB8Ph9MJhOsVmtEX5vNBp/Pp/f5c/hMtU+1Tae+vh4Wi0VfioqKZls2JRJOQqe8WQeQy+XCqVOncOjQoVjWM626ujpomqYvfX19c/6cFAecA0p5s/oyak1NDZqbm3H06FEUFhbq2+12O8bGxjA4OBhxFuT3+2G32/U+7e3tEfubuks21eevzGYzzGbzbEqlBCYYQCkvqjMgIQRqamrQ1NSE1tZWLFy4MKK9tLQUGRkZcLvd+raenh709vZCVVUAgKqq6OzsRCAQ0Pu0tLRAURSUlJRczbFQ0mEApbqozoBcLhcOHjyIzz77DDk5OfqcjcViQWZmJiwWCzZt2oTa2lrk5uZCURRs3boVqqqivLwcALBmzRqUlJTg0Ucfxe7du+Hz+fD888/D5XLxLCfF8AyIogqghoYGAMDdd98dsX3fvn14/PHHAQBvvvkmjEYjqqqqEAqFUFFRgffff1/vm5aWhubmZmzZsgWqqiIrKwvV1dV45ZVXru5IKPlwEjrlGUQSvg0Fg0FYLBZomgZFUWSXQ7N0rucYzhzZr6/rf5As6zp5RVFMzPQ1yu+CkTRJ+N5HMcYAInkYQCmPAUTSCHAOKNUxgEgengGlPAYQycMASnkMIJImHJ6M3GAwADBIqYXkYACRNKP/6o9YN2XnwZjBD6OmEgYQSSP+cgZkNKbBwDOglMIAosRhMPAKLMUwgCiBcA4o1TCAKGEYDAyfVMMAIiJpGECUOHgbPuUwgChxcBI65TCAKGEYeBM+5TCAKHHwEizlMIAocfAuWMphAFHCMIBzQKmGAUSJg5dgKYcBRAmE4ZNqGECUMDgFlHoYQJRADEyhFMMAosRh4OeAUg0DiBIHz35STlS/jEoUjbGxMVy4cOGK7X82MT6BQU2DwTD9+2JmZiZ/vvsawwCiOdPU1ITt27dftt21bglWLyvS192trdj91H9d9m/Vv/rqq9i4cWOsyySJGEA0Z0ZGRnD27NnLtg+P/DvOXFyK38cdyMs4i+EL/42zZ89eNoCGh4fnqFKSJao5oIaGBixduhSKokBRFKiqii+++EJvHx0dhcvlQl5eHrKzs1FVVQW/3x+xj97eXlRWVmL+/PnIz8/Hjh07MDExEZujoaTy68XF+J8LKzEwXoCfL5TiHxeWAvylnpQSVQAVFhZi165d8Hq9OHnyJFatWoX7778fXV1dAIBnnnkGn3/+ORobG9HW1obffvsNDz74oP74yclJVFZWYmxsDMeOHcOBAwewf/9+7Ny5M7ZHRUlhZNICof8TNGJowsr8STFRXYKtW7cuYv31119HQ0MDjh8/jsLCQnzwwQc4ePAgVq1aBQDYt28fbrnlFhw/fhzl5eX4+uuv0d3djW+++QY2mw233norXn31VTz77LN46aWXYDKZYndklPDspjPIMFzEuJiHDEMIdtM/ZJdEcTbrOaDJyUk0NjZiZGQEqqrC6/VifHwcTqdT77No0SI4HA54PB6Ul5fD4/FgyZIlsNlsep+Kigps2bIFXV1duO2226Kq4fTp08jOzp7tIdAc6+/vv2J756mj8J3z41/jdlgzAhg813PF/j6fD93d3bEskebITOfrog6gzs5OqKqK0dFRZGdno6mpCSUlJejo6IDJZILVao3ob7PZ4PP5APzxD+jP4TPVPtV2OaFQCKFQSF8PBoMAAE3TOH+UwK50Cx4AjnX1AV19M97f6OgoBgcHr7IqioeRkZEZ9Ys6gG6++WZ0dHRA0zR88sknqK6uRltbW9QFRqO+vh4vv/zyJdvLysqgKMqcPjfN3unTp2O6v+LiYtxxxx0x3SfNjamThL8T9SehTSYTbrzxRpSWlqK+vh7Lli3D22+/DbvdjrGxsUveofx+P+x2OwDAbrdfcldsan2qz3Tq6uqgaZq+9PXN/F2TiBLXVX8VIxwOIxQKobS0FBkZGXC73XpbT08Pent7oaoqAEBVVXR2diIQCOh9WlpaoCgKSkpKLvscZrNZv/U/tRBR8ovqEqyurg5r166Fw+HA0NAQDh48iCNHjuCrr76CxWLBpk2bUFtbi9zcXCiKgq1bt0JVVZSXlwMA1qxZg5KSEjz66KPYvXs3fD4fnn/+ebhcLn7EnigFRRVAgUAAjz32GPr7+2GxWLB06VJ89dVXuOeeewAAb775JoxGI6qqqhAKhVBRUYH3339ff3xaWhqam5uxZcsWqKqKrKwsVFdX45VXXontUVFCMJlMMT1b5ZvUtccgxOU++J64gsEgLBYLNE3j5VgCGxkZwcDAQMz2Z7VakZOTE7P90dyZ6WuU3wWjOZOVlYWsrCzZZVAC498DIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNKkyy5gNoQQAIBgMCi5EiKaztRrc+q1ejlJGUDnz58HABQVFUmuhIiuZGhoCBaL5bLtSRlAubm5AIDe3t4rHhxFCgaDKCoqQl9fHxRFkV1OUuCYzY4QAkNDQygoKLhiv6QMIKPxj6kri8XCfxSzoCgKxy1KHLPozeTkgJPQRCQNA4iIpEnKADKbzXjxxRdhNptll5JUOG7R45jNLYP4u/tkRERzJCnPgIjo2sAAIiJpGEBEJA0DiIikScoA2rNnD4qLizFv3jyUlZWhvb1ddknS1NfXY8WKFcjJyUF+fj7Wr1+Pnp6eiD6jo6NwuVzIy8tDdnY2qqqq4Pf7I/r09vaisrIS8+fPR35+Pnbs2IGJiYl4Hoo0u3btgsFgwLZt2/RtHLM4EUnm0KFDwmQyiQ8//FB0dXWJzZs3C6vVKvx+v+zSpKioqBD79u0Tp06dEh0dHeK+++4TDodDDA8P632efPJJUVRUJNxutzh58qQoLy8Xd9xxh94+MTEhFi9eLJxOp/jxxx/F4cOHxYIFC0RdXZ2MQ4qr9vZ2UVxcLJYuXSqefvppfTvHLD6SLoBWrlwpXC6Xvj45OSkKCgpEfX29xKoSRyAQEABEW1ubEEKIwcFBkZGRIRobG/U+P/30kwAgPB6PEEKIw4cPC6PRKHw+n96noaFBKIoiQqFQfA8gjoaGhsRNN90kWlpaxF133aUHEMcsfpLqEmxsbAxerxdOp1PfZjQa4XQ64fF4JFaWODRNA/D/X9j1er0YHx+PGLNFixbB4XDoY+bxeLBkyRLYbDa9T0VFBYLBILq6uuJYfXy5XC5UVlZGjA3AMYunpPoy6rlz5zA5ORnxPx0AbDYbTp8+LamqxBEOh7Ft2zbceeedWLx4MQDA5/PBZDLBarVG9LXZbPD5fHqf6cZ0qu1adOjQIfzwww/4/vvvL2njmMVPUgUQXZnL5cKpU6fw3XffyS4lofX19eHpp59GS0sL5s2bJ7uclJZUl2ALFixAWlraJXcj/H4/7Ha7pKoSQ01NDZqbm/Htt9+isLBQ32632zE2NobBwcGI/n8eM7vdPu2YTrVda7xeLwKBAG6//Xakp6cjPT0dbW1teOedd5Ceng6bzcYxi5OkCiCTyYTS0lK43W59WzgchtvthqqqEiuTRwiBmpoaNDU1obW1FQsXLoxoLy0tRUZGRsSY9fT0oLe3Vx8zVVXR2dmJQCCg92lpaYGiKCgpKYnPgcTR6tWr0dnZiY6ODn1Zvnw5NmzYoP83xyxOZM+CR+vQoUPCbDaL/fv3i+7ubvHEE08Iq9UacTcilWzZskVYLBZx5MgR0d/fry8XLlzQ+zz55JPC4XCI1tZWcfLkSaGqqlBVVW+fuqW8Zs0a0dHRIb788ktx/fXXp9Qt5T/fBROCYxYvSRdAQgjx7rvvCofDIUwmk1i5cqU4fvy47JKkATDtsm/fPr3PxYsXxVNPPSWuu+46MX/+fPHAAw+I/v7+iP38+uuvYu3atSIzM1MsWLBAbN++XYyPj8f5aOT5awBxzOKDf46DiKRJqjkgIrq2MICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJp/g8X02lDsla32wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        over = terminated or truncated\n",
    "\n",
    "        #限制最大步数\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            over = True\n",
    "        \n",
    "        #没坚持到最后,扣分\n",
    "        if over and self.step_n < 200:\n",
    "            reward = -1000\n",
    "\n",
    "        return state, reward, over\n",
    "\n",
    "    #打印游戏图像\n",
    "    def show(self):\n",
    "        from matplotlib import pyplot as plt\n",
    "        plt.figure(figsize=(3, 3))\n",
    "        plt.imshow(self.env.render())\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()\n",
    "\n",
    "env.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ecfbe912",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Sequential(\n",
       "   (0): Linear(in_features=4, out_features=64, bias=True)\n",
       "   (1): ReLU()\n",
       "   (2): Linear(in_features=64, out_features=64, bias=True)\n",
       "   (3): ReLU()\n",
       "   (4): Linear(in_features=64, out_features=2, bias=True)\n",
       " ),\n",
       " Sequential(\n",
       "   (0): Linear(in_features=4, out_features=64, bias=True)\n",
       "   (1): ReLU()\n",
       "   (2): Linear(in_features=64, out_features=64, bias=True)\n",
       "   (3): ReLU()\n",
       "   (4): Linear(in_features=64, out_features=2, bias=True)\n",
       " ))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "#定义模型,评估状态下每个动作的价值\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 2),\n",
    ")\n",
    "\n",
    "#延迟更新的模型,用于计算target\n",
    "model_delay = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 2),\n",
    ")\n",
    "\n",
    "#复制参数\n",
    "model_delay.load_state_dict(model.state_dict())\n",
    "\n",
    "model, model_delay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "84cbf0ff",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-991.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "#玩一局游戏并记录数据\n",
    "def play(show=False):\n",
    "    data = []\n",
    "    reward_sum = 0\n",
    "\n",
    "    state = env.reset()\n",
    "    over = False\n",
    "    while not over:\n",
    "        action = model(torch.FloatTensor(state).reshape(1, 4)).argmax().item()\n",
    "        if random.random() < 0.1:\n",
    "            action = env.action_space.sample()\n",
    "\n",
    "        next_state, reward, over = env.step(action)\n",
    "\n",
    "        data.append((state, action, reward, next_state, over))\n",
    "        reward_sum += reward\n",
    "\n",
    "        state = next_state\n",
    "\n",
    "        if show:\n",
    "            display.clear_output(wait=True)\n",
    "            env.show()\n",
    "\n",
    "    return data, reward_sum\n",
    "\n",
    "\n",
    "play()[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7bfcfd21",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_7015/2282743005.py:27: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  ../torch/csrc/utils/tensor_new.cpp:201.)\n",
      "  state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(202,\n",
       " (array([ 0.00668867, -0.00856509,  0.04751302,  0.03165938], dtype=float32),\n",
       "  1,\n",
       "  1.0,\n",
       "  array([ 0.00651737,  0.18584444,  0.04814621, -0.24566224], dtype=float32),\n",
       "  False))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#数据池\n",
    "class Pool:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.pool = []\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.pool)\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        return self.pool[i]\n",
    "\n",
    "    #更新动作池\n",
    "    def update(self):\n",
    "        #每次更新不少于N条新数据\n",
    "        old_len = len(self.pool)\n",
    "        while len(pool) - old_len < 200:\n",
    "            self.pool.extend(play()[0])\n",
    "\n",
    "        #只保留最新的N条数据\n",
    "        self.pool = self.pool[-2_0000:]\n",
    "\n",
    "    #获取一批数据样本\n",
    "    def sample(self):\n",
    "        data = random.sample(self.pool, 64)\n",
    "\n",
    "        state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n",
    "        action = torch.LongTensor([i[1] for i in data]).reshape(-1, 1)\n",
    "        reward = torch.FloatTensor([i[2] for i in data]).reshape(-1, 1)\n",
    "        next_state = torch.FloatTensor([i[3] for i in data]).reshape(-1, 4)\n",
    "        over = torch.LongTensor([i[4] for i in data]).reshape(-1, 1)\n",
    "\n",
    "        return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "pool = Pool()\n",
    "pool.update()\n",
    "pool.sample()\n",
    "\n",
    "len(pool), pool[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "018c4b1e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 408 -950.65\n",
      "100 20000 -171.0\n",
      "200 20000 200.0\n",
      "300 20000 200.0\n",
      "400 20000 145.65\n",
      "500 20000 200.0\n",
      "600 20000 200.0\n",
      "700 20000 200.0\n",
      "800 20000 200.0\n",
      "900 20000 200.0\n"
     ]
    }
   ],
   "source": [
    "#训练\n",
    "def train():\n",
    "    model.train()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=2e-4)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #共更新N轮数据\n",
    "    for epoch in range(1000):\n",
    "        pool.update()\n",
    "\n",
    "        #每次更新数据后,训练N次\n",
    "        for i in range(200):\n",
    "\n",
    "            #采样N条数据\n",
    "            state, action, reward, next_state, over = pool.sample()\n",
    "\n",
    "            #计算value\n",
    "            value = model(state).gather(dim=1, index=action)\n",
    "\n",
    "            #计算target\n",
    "            with torch.no_grad():\n",
    "                target = model_delay(next_state)\n",
    "            target = target.max(dim=1)[0].reshape(-1, 1)\n",
    "            target = target * 0.99 * (1 - over) + reward\n",
    "\n",
    "            loss = loss_fn(value, target)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "        #复制参数\n",
    "        if (epoch + 1) % 5 == 0:\n",
    "            model_delay.load_state_dict(model.state_dict())\n",
    "\n",
    "        if epoch % 100 == 0:\n",
    "            test_result = sum([play()[-1] for _ in range(20)]) / 20\n",
    "            print(epoch, len(pool), test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a101d0e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAS7ElEQVR4nO3df2xT5b8H8PfarWWwnc4O17q79UIiEXZhqAO2I99cvVKZuBDR/aGG6DRcjNgRYIboIj/8mRnMjT9h/KPAP4iZ907DAujcYMQvHZPpcseARaOmC9BW4PZ0G6zd2uf+8c3O18rAdYw+63i/kpPQ5/n09HMe1jftOe1IEUIIEBFJYJDdABHduhhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkjbQA2r59O2bMmIEpU6aguLgYbW1tslohIkmkBNDnn3+OqqoqbN26FT/88APmz5+P0tJS+P1+Ge0QkSQpMr6MWlxcjIULF+Ljjz8GAESjUeTn52Pt2rV45ZVXEt0OEUmSmugHDIfDaG9vR3V1tT5mMBjgdDrhdrtHvE8oFEIoFNJvR6NRXLp0CdnZ2UhJSbnpPRNRfIQQ6O3tRW5uLgyGa7/RSngAXbhwAZFIBDabLWbcZrPhzJkzI96npqYGr7/+eiLaI6Jx1NPTg7y8vGvOJzyAxqK6uhpVVVX6bU3T4HA40NPTA0VRJHZGRCMJBoPIz89HZmbmdesSHkDTp0+H0WiEz+eLGff5fLDb7SPex2w2w2w2XzWuKAoDiGgC+6tTJAm/CmYymVBUVISmpiZ9LBqNoqmpCaqqJrodIpJIyluwqqoqVFRUYMGCBVi0aBHef/999Pf347nnnpPRDhFJIiWAnnjiCfz+++/YsmULvF4v7r77bhw6dOiqE9NENLlJ+RzQjQoGg7BYLNA0jeeAiCag0T5H+V0wIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARSRN3AB09ehTLly9Hbm4uUlJS8OWXX8bMCyGwZcsW3HHHHUhPT4fT6cRPP/0UU3Pp0iWsXLkSiqIgKysLq1atQl9f3w0dCBEln7gDqL+/H/Pnz8f27dtHnN+2bRs+/PBD7Ny5E8ePH8e0adNQWlqKgYEBvWblypXo6upCY2MjGhoacPToUTz//PNjPwoiSk7iBgAQ9fX1+u1oNCrsdrt499139bFAICDMZrP47LPPhBBCnDp1SgAQ33//vV5z8OBBkZKSIs6ePTuqx9U0TQAQmqbdSPtEdJOM9jk6rueAfv31V3i9XjidTn3MYrGguLgYbrcbAOB2u5GVlYUFCxboNU6nEwaDAcePHx9xv6FQCMFgMGYjouQ3rgHk9XoBADabLWbcZrPpc16vFzk5OTHzqampsFqtes2f1dTUwGKx6Ft+fv54tk1EkiTFVbDq6mpomqZvPT09slsionEwrgFkt9sBAD6fL2bc5/Ppc3a7HX6/P2Z+aGgIly5d0mv+zGw2Q1GUmI2Ikt+4BtDMmTNht9vR1NSkjwWDQRw/fhyqqgIAVFVFIBBAe3u7XtPc3IxoNIri4uLxbIeIJrjUeO/Q19eHn3/+Wb/966+/oqOjA1arFQ6HA+vXr8dbb72FWbNmYebMmdi8eTNyc3OxYsUKAMCcOXPw8MMPY/Xq1di5cycGBwdRWVmJJ598Erm5ueN2YESUBOK9vHb48GEB4KqtoqJCCPGPS/GbN28WNptNmM1msWTJEtHd3R2zj4sXL4qnnnpKZGRkCEVRxHPPPSd6e3vH/RIfEckx2udoihBCSMy/MQkGg7BYLNA0jeeDiCag0T5Hk+IqGBFNTgwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISJq4/1semvyEiELznEQkfEUfm3b7DEzJsl3nXkTxYwDRVUQ0gp7j/4OB/zunjzn+9hQDiMYd34IRkTQMICKShgFERNIwgIhIGgYQEUnDACIiaeIKoJqaGixcuBCZmZnIycnBihUr0N3dHVMzMDAAl8uF7OxsZGRkoLy8HD6fL6bG4/GgrKwMU6dORU5ODjZu3IihoaEbPxoiSipxBVBLSwtcLhdaW1vR2NiIwcFBLF26FP39/XrNhg0bsH//ftTV1aGlpQXnzp3D448/rs9HIhGUlZUhHA7j2LFj2LNnD3bv3o0tW7aM31ERUXIQN8Dv9wsAoqWlRQghRCAQEGlpaaKurk6vOX36tAAg3G63EEKIAwcOCIPBILxer15TW1srFEURoVBoVI+raZoAIDRNu5H26RoiQ2Hxv59vFW07V+ub92Sz7LYoiYz2OXpD54A0TQMAWK1WAEB7ezsGBwfhdDr1mtmzZ8PhcMDtdgMA3G435s2bB5vtn5+qLS0tRTAYRFdX14iPEwqFEAwGYzYiSn5jDqBoNIr169dj8eLFmDt3LgDA6/XCZDIhKysrptZms8Hr9eo1fwyf4fnhuZHU1NTAYrHoW35+/ljbJqIJZMwB5HK5cPLkSezbt288+xlRdXU1NE3Tt56enpv+mER0843py6iVlZVoaGjA0aNHkZeXp4/b7XaEw2EEAoGYV0E+nw92u12vaWtri9nf8FWy4Zo/M5vNMJvNY2mViCawuF4BCSFQWVmJ+vp6NDc3Y+bMmTHzRUVFSEtLQ1NTkz7W3d0Nj8cDVVUBAKqqorOzE36/X69pbGyEoigoKCi4kWMhoiQT1ysgl8uFvXv34quvvkJmZqZ+zsZisSA9PR0WiwWrVq1CVVUVrFYrFEXB2rVroaoqSkpKAABLly5FQUEBnn76aWzbtg1erxebNm2Cy+XiqxyiW0xcAVRbWwsAeOCBB2LGd+3ahWeffRYA8N5778FgMKC8vByhUAilpaXYsWOHXms0GtHQ0IA1a9ZAVVVMmzYNFRUVeOONN27sSIgo6aQIIYTsJuIVDAZhsVigaRoURZHdzqQTjQyi67/fvuoXktn+7T8kdkXJZLTPUX4XjIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJE1c/zMqTR6XL19GOBwecU5EhhCNRGLGrly+gkAgcM39ZWRkIDWVP04UH/7E3KJeffVV1NXVjTiXajTgv/5zMfJvz9TH3nr7LRw64Rmx3mg0Yv/+/SgsLLwpvdLkxQC6RQUCAZw9e3bEuTSjAf3hVJzqX4z+SBbyppxBIPD3a9YbjUYMDg7ezHZpkorrHFBtbS0KCwuhKAoURYGqqjh48KA+PzAwAJfLhezsbGRkZKC8vBw+ny9mHx6PB2VlZZg6dSpycnKwceNGDA0Njc/R0LgQMKCr72/wDBTg4uC/4GTvv+P3cL7stmgSiiuA8vLy8M4776C9vR0nTpzAgw8+iEcffRRdXV0AgA0bNmD//v2oq6tDS0sLzp07h8cff1y/fyQSQVlZGcLhMI4dO4Y9e/Zg9+7d2LJly/geFd0QgRT0R7IApAAAIkjDlUjmde9DNBZxvQVbvnx5zO23334btbW1aG1tRV5eHj755BPs3bsXDz74IABg165dmDNnDlpbW1FSUoJvvvkGp06dwrfffgubzYa7774bb775Jl5++WW89tprMJlM43dkNGYpiCLX/BOi+FdEYcQ0YwDWtHOy26JJaMzngCKRCOrq6tDf3w9VVdHe3o7BwUE4nU69Zvbs2XA4HHC73SgpKYHb7ca8efNgs9n0mtLSUqxZswZdXV2455574urhzJkzyMjIGOsh3NKud0UrGo2i1b0XQ6Y2XI4oyDF58Jun+7r7++WXX5Cenj7OXVKy6uvrG1Vd3AHU2dkJVVUxMDCAjIwM1NfXo6CgAB0dHTCZTMjKyoqpt9ls8Hq9AACv1xsTPsPzw3PXEgqFEAqF9NvBYBAAoGkazx+N0bUuwQNAVAjs//tpAKdHvb++vr7rhhrdWvr7+0dVF3cA3XXXXejo6ICmafjiiy9QUVGBlpaWuBuMR01NDV5//fWrxouLi6Eoyk197MkqJydnXPdXWFiIoqKicd0nJa/hFwl/Je5PQptMJtx5550oKipCTU0N5s+fjw8++AB2ux3hcPiqfwV9Ph/sdjsAwG63X3VVbPj2cM1IqquroWmavvX09MTbNhFNQDf8VYxoNIpQKISioiKkpaWhqalJn+vu7obH44GqqgAAVVXR2dkJv9+v1zQ2NkJRFBQUFFzzMcxms37pf3gjouQX11uw6upqLFu2DA6HA729vdi7dy+OHDmCr7/+GhaLBatWrUJVVRWsVisURcHatWuhqipKSkoAAEuXLkVBQQGefvppbNu2DV6vF5s2bYLL5YLZbL4pB0hEE1dcAeT3+/HMM8/g/PnzsFgsKCwsxNdff42HHnoIAPDee+/BYDCgvLwcoVAIpaWl2LFjh35/o9GIhoYGrFmzBqqqYtq0aaioqMAbb7wxvkdFfyk9PX3cXkmmpqbCaDSOy77o1pIihBCym4hXMBiExWKBpml8OzZGly5dGvWVitGw2Wz8HBfpRvsc5XfBblFWqxVWq1V2G3SL4+8DIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNKkym5gLIQQAIBgMCi5EyIayfBzc/i5ei1JGUAXL14EAOTn50vuhIiup7e3FxaL5ZrzSRlAVqsVAODxeK57cBQrGAwiPz8fPT09UBRFdjtJgWs2NkII9Pb2Ijc397p1SRlABsM/Tl1ZLBb+UIyBoihctzhxzeI3mhcHPAlNRNIwgIhImqQMILPZjK1bt8JsNstuJalw3eLHNbu5UsRfXScjIrpJkvIVEBFNDgwgIpKGAURE0jCAiEiapAyg7du3Y8aMGZgyZQqKi4vR1tYmuyVpampqsHDhQmRmZiInJwcrVqxAd3d3TM3AwABcLheys7ORkZGB8vJy+Hy+mBqPx4OysjJMnToVOTk52LhxI4aGhhJ5KNK88847SElJwfr16/UxrlmCiCSzb98+YTKZxKeffiq6urrE6tWrRVZWlvD5fLJbk6K0tFTs2rVLnDx5UnR0dIhHHnlEOBwO0dfXp9e88MILIj8/XzQ1NYkTJ06IkpIScd999+nzQ0NDYu7cucLpdIoff/xRHDhwQEyfPl1UV1fLOKSEamtrEzNmzBCFhYVi3bp1+jjXLDGSLoAWLVokXC6XfjsSiYjc3FxRU1MjsauJw+/3CwCipaVFCCFEIBAQaWlpoq6uTq85ffq0ACDcbrcQQogDBw4Ig8EgvF6vXlNbWysURRGhUCixB5BAvb29YtasWaKxsVHcf//9egBxzRInqd6ChcNhtLe3w+l06mMGgwFOpxNut1tiZxOHpmkA/vmF3fb2dgwODsas2ezZs+FwOPQ1c7vdmDdvHmw2m15TWlqKYDCIrq6uBHafWC6XC2VlZTFrA3DNEimpvox64cIFRCKRmL90ALDZbDhz5oykriaOaDSK9evXY/HixZg7dy4AwOv1wmQyISsrK6bWZrPB6/XqNSOt6fDcZLRv3z788MMP+P7776+a45olTlIFEF2fy+XCyZMn8d1338luZULr6enBunXr0NjYiClTpshu55aWVG/Bpk+fDqPReNXVCJ/PB7vdLqmriaGyshINDQ04fPgw8vLy9HG73Y5wOIxAIBBT/8c1s9vtI67p8Nxk097eDr/fj3vvvRepqalITU1FS0sLPvzwQ6SmpsJms3HNEiSpAshkMqGoqAhNTU36WDQaRVNTE1RVldiZPEIIVFZWor6+Hs3NzZg5c2bMfFFREdLS0mLWrLu7Gx6PR18zVVXR2dkJv9+v1zQ2NkJRFBQUFCTmQBJoyZIl6OzsREdHh74tWLAAK1eu1P/MNUsQ2WfB47Vv3z5hNpvF7t27xalTp8Tzzz8vsrKyYq5G3ErWrFkjLBaLOHLkiDh//ry+Xb58Wa954YUXhMPhEM3NzeLEiRNCVVWhqqo+P3xJeenSpaKjo0McOnRI3H777bfUJeU/XgUTgmuWKEkXQEII8dFHHwmHwyFMJpNYtGiRaG1tld2SNABG3Hbt2qXXXLlyRbz44ovitttuE1OnThWPPfaYOH/+fMx+fvvtN7Fs2TKRnp4upk+fLl566SUxODiY4KOR588BxDVLDP46DiKSJqnOARHR5MIAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpPl/i8k62/ZrfwgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "play(True)[-1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pt39]",
   "language": "python",
   "name": "conda-env-pt39-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
